const path = require('path')
const fs = require('fs')
const _ = require('lodash')

const Configuration = require('./configuration')
const Request = require('./request')


module.exports = class Crawler {
    constructor(configuration) {
        this.configuration = _.defaultTo(configuration, new Configuration({}))

        this.logger = require('./extensions/logging')(this.configuration).child({label: 'crawler'})

        const options = {logger: this.logger}
        this.dupefilter = require('./extensions/dupefilter')(this.configuration, options)
        this.pipeline = require('./extensions/pipeline')(this.configuration, options)
        this.downloader = require('./downloader')(this.configuration, options)

        const singleton = require('./singleton')
        singleton.logger = this.logger
        singleton.crawler = this
    }

    pipe(name, callback) {
        this.pipeline.pipe(name, callback)
        return this
    }

    run() {
        require('yargs').command('crawl [spider]', 'Run a spider', (yargs) => {
            yargs.positional('spider', {
                describe: 'spider name',
                type: 'string',
            })
        }, (argv) => {
            let spider = this.loadSpider(argv.spider)
            this.runSpider(spider)
        }).command('fetch [url]', 'Fetch a URL using downloader and print its content to stdout', (yargs) => {
            yargs.positional('url', {})
        }, (argv) => {
            //todo
        }).argv
    }

    loadSpider(name) {
        const jspath = path.join(process.cwd(), this.configuration.spidersDir, `${name}.js`)
        if (!fs.existsSync(jspath)) {
            this.logger.error(`spider js file not exists at ${jspath}`)
            process.exit(1)
        }

        const SpiderClass = require(jspath)
        return new SpiderClass(this)
    }

    runSpider(spider) {
        this.logger.info(`start spider ${spider.constructor.name}`)

        spider.startRequests().forEach(r => {
            this.processRequest(r, {spider})
        })
    }

    processRequest(request, {spider}) {
        if (!spider.allow(request)) {
            return
        }

        if (this.dupefilter.filter(request)) {
            this.logger.verbose(`ignore url: ${request.url}`, {label: 'filter'})
            return
        }

        this.downloader(request, (response) => {
            this.processResponse(response, {request, spider})
        }, (error) => {
            //console.log(error)
        })
    }

    processResponse(response, {request, spider}) {
        const parseFn = _.defaultTo(request.callback, spider.parse).bind(spider)

        response.follow = (url, options) => {
            let r = null;
            if (url instanceof Request) {
                r = url
            } else {
                r = new Request(response.urljoin(url), {})
            }

            _.assign(r, options)
            r.referer = _.defaultTo(r.referer, request.url)
            r.attachment = _.defaultTo(r.attachment, request.attachment)

            this.processRequest(r, {spider})
        }

        parseFn(response, {
            follow: response.follow,
            collect: (item) => {
                this.processItem(item, {spider, response, spider})
            },
        })
    }

    processItem(item, {request, response, spider}) {
        return this.pipeline.process(item, {request, response, spider})
    }
}
